import sys
sys.path.append('C:\Program Files\Tinysoft\Analyse.NET')
import TSLPy3 as ts
import info
import tsbytestostr as tr
import pandas as pd
import datetime
import os
import numpy as np
# import uqer_enterprise
import re
import math
import paramiko
import stat
import string
import random

class GetExternalDataSource_CLIENT:
    """
    get external data source
    """
    def __init__(self):
        """
        载入常用数据
        """
        self.root_path = info.DATAPATH.dataRootPath # 地址信息
        self.server_root_path = info.DATAPATH.server_root_path  # 服务器地址
        self.t, self.sftp =self.get_server()  # 与服务器建立连接

    def get_server(self, hostname='172.16.0.11', username='root', password='111111', port=22):
        """
        与服务器建立连接
        :param hostname: 服务器地址
        :param username: 用户名
        :param password: 密码
        :param port: 接口
        :return:
        """
        t = paramiko.Transport((hostname, port))
        t.connect(username=username, password=password)
        sftp = paramiko.SFTPClient.from_transport(t)
        return t, sftp

    def mkdir_p(self, sftp, remote, is_dir=False):
        """
        emulates mkdir_p if required.
        sftp - is a valid sftp object
        remote - remote path to create.
        """
        dirs_ = []
        if is_dir:
            dir_ = remote
        else:
            dir_, basename = os.path.split(remote)
        while len(dir_) > 1:
            dirs_.append(dir_)
            dir_, _ = os.path.split(dir_)

        if len(dir_) == 1 and not dir_.startswith("/"):
            dirs_.append(dir_)  # For a remote path like y/x.txt

        while len(dirs_):
            dir_ = dirs_.pop()
            try:
                sftp.stat(dir_)
            except:
                print("making ... dir", dir_)
                sftp.mkdir(dir_)

    def put_file(self, local_path, remote_path):
        """
        向服务器传输文件
        :param local_path: 本机地址
        :param remote_path: 远程地址
        :return:
        """
        self.mkdir_p(self.sftp, remote_path)
        self.sftp.put(local_path, remote_path)


    def get_file(self, filename, remote_path):
        """
        :param filename: 文件名称
        :param remote_path: 远程地址
        :return: 
        
        从远程服务器获取数据
        """
        raw_pwd = ''.join(random.sample('abcdefghijklmnopqrstuvwxyz1234567890', 16)).replace(' ', '')
        local_path = 'tmp\\' + raw_pwd + '.pkl'
        self.sftp.get(remote_path, local_path)
        file = pd.read_pickle(local_path)
        os.remove(local_path)
        return file

    def _get_all_files_in_remote_dir(self, sftp, remote_dir):
        all_files = list()
        if remote_dir[-1] == '/':
            remote_dir = remote_dir[0:-1]

        files = sftp.listdir_attr(remote_dir)
        for file in files:
            filename = remote_dir + '/' + file.filename

            if stat.S_ISDIR(file.st_mode):  # 如果是文件夹的话递归处理
                all_files.extend(self._get_all_files_in_remote_dir(sftp, filename))
            else:
                all_files.append(filename)

        return all_files

    # 本地文件夹的上传到远程服务器
    def sftp_get_dir(self, remote_dir, local_dir):
        try:

            sftp = paramiko.SFTPClient.from_transport(self.t)

            all_files = self._get_all_files_in_remote_dir(sftp, remote_dir)

            for file in all_files:

                local_filename = file.replace(remote_dir, local_dir)
                local_filepath = os.path.dirname(local_filename)

                if not os.path.exists(local_filepath):
                    os.makedirs(local_filepath)

                sftp.get(file, local_filename)

        except:
            print('ssh get dir from master failed.')

    def download_basic_data(self, ):
        """
        获取日历和基本信息
        日历: 20100101-最新交易日
        基本信息:
            ProductNames: 全部商品名称
            ContractInfo: 合约基本信息
            PriceRate: 复权因子
            ZL: 主力合约名称
        :return:
        """
        # 获取日历
        self.sftp_get_dir(info.DATAPATH.server_root_path + '/' + 'calendar', info.DATAPATH.dataRootPath + '\\' + 'calendar')
        # 获取基础数据, 包括全部产品名称, 合约信息, 主力合约, 复权因子
        self.sftp_get_dir(info.DATAPATH.server_root_path + '/' + 'future_basic_data', info.DATAPATH.dataRootPath + '\\' + 'future_basic_data')
        # 获取日线
        # self.sftp_get_dir(info.DATAPATH.server_root_path + '/' + 'future_basic_data' + '/' + 'dailyBar',
        #                   info.DATAPATH.dataRootPath + '\\' + 'future_basic_data' + '\\' + 'dailyBar')

    def get_ZL_data(self, freq_name='dailyBar_ZL', product_name='中证500指数_1D'):
        ZL_data = self.get_file(product_name, remote_path=info.DATAPATH.server_root_path + '/' + 'future_trade_data' + '/' + freq_name + '/' + product_name + '.pkl')
        return ZL_data

    def get_remote_backtest_data_info(self):
        print('Daily Data')
        fileNames = self.sftp.listdir(info.DATAPATH.server_root_path + '/' + 'future_trade_data' + '/' + 'dailyBar_ZL')
        for fileName in fileNames:
            df = self.get_file(fileName, info.DATAPATH.server_root_path + '/' + 'future_trade_data' + '/' + 'dailyBar_ZL' + '/' + fileName)
            print(fileName, df.index[0], df.index[-1])
        print('High Frequent Data')
        fileNames = self.sftp.listdir(info.DATAPATH.server_root_path + '/' + 'future_trade_data' + '/' + 'hfBar_ZL')
        for fileName in fileNames:
            df = self.get_file(fileName,
                               info.DATAPATH.server_root_path + '/' + 'future_trade_data' + '/' + 'hfBar_ZL' + '/' + fileName)
            print(fileName, df.index[0], df.index[-1])

if __name__ == '__main__':

    a = GetExternalDataSource_CLIENT()
    # 下载基础数据
    a.download_basic_data()

    #df = a.get_ZL_data(freq_name='dailyBar_ZL', product_name='中证500指数_1D')

    print(a.get_remote_backtest_data_info())